#!/usr/bin/env python3
#########################################################################
# 本脚本用于FileCoin日常巡检，及时告警通知到群
#
#
#
#########################################################################

from __future__ import print_function
from math import fabs
from multiprocessing.dummy import current_process
import time
import os
import re
import sys
import subprocess as sp
import socket
from environs import Env
import traceback
from functools import lru_cache
from DingTalkMsg import DingTalkMsg

global env
global block_count  #初始暴块数
env = Env()
env.read_env()

def print(s, end='\n', file=sys.stdout):
    file.write(s + end)
    file.flush()


def is_number(s):
    try:
        float(s)
        return True
    except ValueError:
        pass

    try:
        import unicodedata
        unicodedata.numeric(s)
        return True
    except (TypeError, ValueError):
        pass
    return False


def send_msg(content='推送默认内容', isAtAll=False):
    machine_name = socket.gethostname()
    webhookUrl = env.str("dingding_webhook", "")
    secret = env.str("dingding_secret", "")
    if not webhookUrl or not secret:
        print("dingding_webhook或dingding_secret未配置，推送消息失败")
        return False
    content = current_ip + ":" + machine_name + "，" + content
    #进行告警通知的时间区间
    time_range = env.list("send_alert_time", [10,17])
    now_hour = time.strftime("%H", time.localtime())
    can_send =  True if now_hour in set(time_range) else False
    try:
        ding = DingTalkMsg(webhookUrl, secret)
        if can_send:
            ding.sendMsg(content, isAtAll)
        print("can_send=%s,send msg:%s"%(can_send, content))
    except Exception as e:
        print("Fail to send message: " + e)

# 获取当前机器局域网IP
def get_ip():
    try:
        out = sp.getoutput("ip addr | awk -F \"[ /]+\" '/inet /{print $3}' | grep -v '127.0.0.1' | head -n 1")
        print('get_current_ip:')
        print(out)
        ip = out.strip()
        if not ip:
            send_msg("获取当前机器IP出错，脚本无法继续执行")
            return False
        return ip
    except Exception as e:
        print("Fail to get_ip: " + e)

# 高度同步检查
def chain_check():
    try:
        out = sp.getoutput("timeout 36s lotus sync wait")
        print('chain_check:')
        print(out)
        if out.endswith('Done!'):
            print("true")
            return True
        send_msg("节点同步出错，请及时排查！")
        return False
    except Exception as e:
        print("Fail to send message: " + e)

# 获取主机显卡数
def get_nvidia_num():
    out = sp.getoutput("timeout 30s echo $(nvidia-smi | grep GeForce | wc -l)")
    print('get_nvidia_num=%s'%(out))
    return int(out)

# 显卡驱动检查
def nvidia_check():
    nvidia_num = get_nvidia_num()
    if nvidia_num > 0:
        return True
    send_msg("显卡驱动故障，请及时排查！")
    return False

# miner进程检查
def minerprocess_check():
    out = sp.getoutput("timeout 30s echo $(pidof lotus-miner)")
    print('minerprocess_check:')
    print(out)
    if out.strip():
        print("true")
        return True
    send_msg("Miner进程丢失，请及时排查！")
    return False


# worker进程检查
def worker_check(isC1 = False):
    out = sp.getoutput("timeout 30s echo $(pidof lotus-worker)")
    print('workerprocess_check:')
    print(out)
    out = out.strip()
    if not out and not isC1:
        send_msg("Worker进程丢失，请及时排查！")
        return False
    #c1 worker需要检查显卡数和work数是否一致
    if isC1:
        worker_num = 0 if not out else len(out.split(' '))
        nvidia_num = get_nvidia_num()
        print("worker_num=%d,nvidia_num=%d"%(worker_num, nvidia_num))
        if worker_num < nvidia_num:
            c2_path = env.str("start_c2_path", "")
            if c2_path:
                start_c2_work(nvidia_num)
            else:
                c2_path = "未配置start_c2.sh路径，暂无法自启"
            send_msg("Worker数%d和机器显卡数%d不一致，正在自动执行 %s..."%(worker_num, nvidia_num, c2_path))
            return False
    return True

def start_c2_work(total_num):
    c2_path = env.str("start_c2_path", "")
    if not c2_path:
        print("start_c2_path 配置为空，无法执行")
        return False
    for i in range(total_num):
        num = i + 1
        out = sp.getoutput("timeout 30s echo $(sh %s %d %d)"%(c2_path, ip_tail, num))
        print('exec sh %s %d %d >>>>'%(c2_path, ip_tail, num))
        print(out)
    return True

def p_worker_check():
    return worker_check(False)

def c_worker_check():
    return worker_check(True)

# lotus进程检查
def lotusprocess_check():
    out = sp.getoutput("timeout 30s echo $(pidof lotus)")
    print('lotusprocess_check:')
    print(out)
    if out.strip():
        print("true")
        return True
    send_msg("Lotus进程丢失，请及时排查！")
    print("false")
    return False


# 消息堵塞检查
def mpool_check():
    # lotus mpool pending --local |grep Nonce |wc -l
    out = sp.getoutput("lotus mpool pending --local | wc -l")
    print('mpool_check:')
    print(out)
    if is_number(out):
        if int(out) <= 240:
            print("true")
            return True
        send_msg("消息堵塞，请及时清理！")
    return False


# 存储文件挂载检查，磁盘容量剩余检查
def fm_check(file_mount=env.list("file_mount"),
             disk_avail_alert=env.int("disk_avail_alert")):
    if not file_mount:
        print("请填写巡检挂载磁盘路径file_mount")
        return False
    is_fm_correct = True
    for str in file_mount:
        out = sp.getoutput("timeout 30s echo $(df -hl | grep -w  " + str +
                           " | awk '{print $4}'" + ")")
        print('fm_check:')
        print(out)
        if not out.strip():
            print("false")
            send_msg("未发现存储挂载目录%s，请及时排查！"%(str))
            is_fm_correct = False
        if out.find('G') >= 0 and int(
                out[0:out.find('G')]) <= disk_avail_alert:
            print("false")
            send_msg("%s磁盘空间不足，请及时排查！"%(str))
            is_fm_correct = False
    return is_fm_correct


# WindowPost—Miner日志报错检查
def wdpost_log_check(wdpost_log_path=env.str("wdpost_log_path")):
    if not wdpost_log_path:
        print("请填写WindowPost—Miner日志路径 wdpost_log_path")
        return False
    out = sp.getoutput("cat " + wdpost_log_path +
                       "| grep 'running window post failed'")
    print('wdpost_log_check:')
    print(out)
    if not out.strip():
        print("true")
        return True
    send_msg("Wdpost报错，请及时处理！")
    return False


# # 获取爆块数；加缓存是为了获取上次的结果，获取最新爆块数需要先执行get_block_count.cache_clear()
# @lru_cache()
# def get_block_count():
#     winingpost_log_path = env.str("winingpost_log_path")
#     if not winingpost_log_path:
#         print("请填写winingpost_log_path")
#         return -1
#     out = sp.getoutput("cat "+ winingpost_log_path +" | grep 'mined new block' | wc -l")
#     print("get_block_count:")
#     return int(out)

# WiningPost—Miner爆块检查
# def mined_block_check(fil_account, check_interval=300):
#     if not fil_account:
#         print("请填写Miner节点号fil_account")
#         return False
#     mined_block_cmd = "lotus chain list --count {0} |grep {1} |wc -l".format(int(check_interval/30), fil_account)
#     out = sp.getoutput(mined_block_cmd)
#     print('mined_block_check:')
#     print(out)
#     block_count=int(out)
#     if block_count > 0:
#         print("true")
#         send_msg("{0}又爆了{1}个块".format(fil_account, block_count)+"，大吉大利，今晚吃鸡")
#         return True
#     return False


# P1任务超时检查
def p1_overtime_check(p1_job_time_alert=env.int("p1_job_time_alert", 5)):
    out = sp.getoutput(
        "lotus-miner sealing jobs | grep -w PC1 | awk '{ print $5\",\"$8}' | head -n 1 | tail -n 1"
    )
    print('overtime_check:')
    print(out)
    out = out.split(',')
    hostname = out[0]
    out = out[1]
    if (out.find("Time") >= 0) or (not out.find('h') >= 0):
        print("time true")
        return True
    if out.strip() and int(out[0:out.find('h')]) <= p1_job_time_alert:
        print(out[0:out.find("h")])
        print("true")
        return True
    send_msg("%s P1封装任务超过%dh，请及时处理！" % (hostname, p1_job_time_alert))
    return False


# P2任务超时检查
def p2_overtime_check(p2_job_time_alert=env.int("p2_job_time_alert", 40)):
    out = sp.getoutput(
        "lotus-miner sealing jobs | grep -w PC2 | awk '{ print $5\",\"$8}' | head -n 1 | tail -n 1"
    )
    print('overtime_check:')
    print(out)
    out = out.split(',')
    hostname = out[0]
    out = out[1].strip()
    if (out.find('m') >= 0):
        if int(out[0:out.find('m')]) <= p2_job_time_alert:
            send_msg("%s P2封装任务超过%dmin，请及时处理！" % (hostname, p2_job_time_alert))
            return False
    elif (out.find('h') >= 0):
        send_msg("%s P2封装任务超过1h，请及时处理！" % (hostname))
        return False

    return True


# C2任务超时检查
def c2_overtime_check(c2_job_time_alert=env.int("c2_job_time_alert", 25)):
    out = sp.getoutput(
        "lotus-miner sealing jobs | grep -w C2 | awk '{ print $5\",\"$8}' | head -n 1 | tail -n 1"
    )
    print('overtime_check:')
    print(out)
    out = out.split(',')
    hostname = out[0]
    out = out[1]
    if (out.find("Time") >= 0) or (not out.find('m') >= 0):
        print("time true")
        return True
    if out.strip() and int(out[0:out.find('m')]) <= c2_job_time_alert:
        print(out[0:out.find("m")])
        print("true")
        return True
    send_msg("%s C2封装任务超过%dmin，请及时处理！" % (hostname, c2_job_time_alert))
    return False


# Default钱包余额预警
def balance_check(min_val=env.float("default_wallet_balance_min", 30), max_val=env.float("default_wallet_balance_max", 0)):
    out = sp.getoutput("lotus wallet balance")
    print('balance_check:')
    print(out)
    balance = out.split(' ')
    if is_number(balance[0]):
        balance = float(balance[0])
        if balance < float(min_val):
            send_msg("当前钱包余额：%s，请尽快充值"%(str(balance)), False)
            return False
        if balance > float(max_val) and float(max_val)>0:
            send_msg("当前钱包余额：%s，已超出设定上限值%s"%(str(balance), str(max_val)), False)
            return False
    else:
        print("无法获取默认钱包余额，将跳过余额检测告警")
    return True

# ssh 登录IP是否授权检查
def ssh_login_ip_check(ssh_white_ip_list=env.list("ssh_white_ip_list", [])):
    if not ssh_white_ip_list:
        print('ssh_white_ip_list配置为空，跳过登录IP授权检测')
        return False
    try:
        print("ssh logined ip check:\n")
        hostname = sp.getoutput("hostname")
        #获取已登录用户IP地址列表
        out = sp.getoutput("who |grep -v tmux |awk '{print $5}'")
        out = out.replace("(", "").replace(")", "")
        login_ip_list = out.split('\n')
        #去除重复
        login_ip_list = set(login_ip_list)
        login_ip_list = list(login_ip_list)
        #检测已登录IP是否授权
        for ip in login_ip_list:
            if ip != "" and ip not in ssh_white_ip_list:
                curtime = time.strftime('%Y-%m-%d %H:%M:%S',
                                        time.localtime(time.time()))
                msg = "{0},未授权IP:{1},已登录服务器{2}".format(curtime, ip, hostname)
                send_msg(hostname + msg)
        print("--------ssh logined ip check finished -------------")
    except Exception as e:
        print(str(e))


# 扇区证明出错检查
def sectors_fault_check(fil_account=env.str("fil_account")):
    if not fil_account:
        print('fil_account配置为空，跳过扇区证明出错检查')
        return False
    sectors_fault_cmd = "lotus-miner proving faults|wc -l"
    out = sp.getoutput(sectors_fault_cmd)
    print('sectors_fault_check:')
    print(out)
    sectors_count = int(out)
    if sectors_count > 2:
        print("true")
        send_msg("{0}节点出错{1}个扇区".format(fil_account, sectors_count - 2) +
                 "，请及时处理")
        return False
    return True

# 检查内网服务器是否可达（宕机或网络不通）
def reachable_check(server_ip=env.list("server_ip", [])):
    if not server_ip:
        print('server_ip配置为空，暂不检测内网网络')
        return False
    try:
        is_reachable = True
        print('reachable_check:')
        for ip in server_ip:
            print("ping %s ... " %(ip))
            p = sp.Popen(["ping -c 4 -W 1 " + ip],
                         stdout=sp.PIPE,
                         stderr=sp.PIPE,
                         shell=True)
            out = p.stdout.read()
            regex = re.compile('100% packet loss')
            if len(regex.findall(str(out))) != 0:
                print("false")
                send_msg("ping %s不可达（宕机/网络故障），请及时排查！"%(ip), False)
                is_reachable = False
            time.sleep(1)
        return is_reachable
    except:
        print('reachable_check error!')

# 检查公网服务器是否可达
def net_check(net_ip=env.list("net_ip", [])):
    if not net_ip:
        print('net_ip配置为空，暂不检测公网网络')
        return False
    is_ip_reach = True
    for str in net_ip:
        out = sp.getoutput("timeout 5s nc -zv " + str)
        print('net_check:')
        print(out)
        if out.find('succeeded') >= 0:
            print("true")
        else:
            print("false")
            send_msg("ping %s不可达（宕机/网络故障），请及时排查！"%(str))
            is_ip_reach = False
        time.sleep(1)
    return is_ip_reach

# 根据机器分组，执行对应的检查方法
def group_fn_exec():
    group_json = env.json("machine_group_json", {})
    for key,ips in group_json.items():
        if ip_tail in set(ips):
            method_list = env.list(key, [])
            if not method_list:
                continue
            print("%s机器巡检开始..."%(key))
            for fn in iter(method_list):
                try:
                    print("%s方法执行..."%(fn))
                    eval(fn)()
                except Exception as e:
                    print("%s方法执行出错：%s" %(fn, e))
                    continue
    return True           

def main():
    while True:
        try:
            global current_ip
            global ip_tail
            current_ip = get_ip()
            if not current_ip:
                return False
            ip_tail = int(current_ip[current_ip.rfind(".") + 1:])
            if reachable_check():
                print("各服务器均可达，无异常")
            if net_check():
                print("各公网均可达，无异常")
            if fm_check():
                print("目录挂载、磁盘空间充足，无异常")

            if not group_fn_exec():
                print("group_fn_exec方法执行出错")

            print("---------------------")
            print(time.asctime(time.localtime(time.time())))
            sleep_time = env.int("check_interval", 900)
            print("巡检完毕sleep %d seconds\n"%sleep_time) 
            time.sleep(sleep_time)
        except KeyboardInterrupt:
            exit(0)
        except:
            traceback.print_exc()
            time.sleep(120)


if __name__ == "__main__":
    main()
